



<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      <meta http-equiv="x-ua-compatible" content="ie=edge">
      
      
      
      
        <meta name="lang:clipboard.copy" content="Copy to clipboard">
      
        <meta name="lang:clipboard.copied" content="Copied to clipboard">
      
        <meta name="lang:search.language" content="en">
      
        <meta name="lang:search.pipeline.stopwords" content="True">
      
        <meta name="lang:search.pipeline.trimmer" content="True">
      
        <meta name="lang:search.result.none" content="No matching documents">
      
        <meta name="lang:search.result.one" content="1 matching document">
      
        <meta name="lang:search.result.other" content="# matching documents">
      
        <meta name="lang:search.tokenizer" content="[\s\-]+">
      
      <link rel="shortcut icon" href="../assets/images/favicon.png">
      <meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
    
    
      
        <title>WMT - sotabencheval Docs</title>
      
    
    
      <link rel="stylesheet" href="../assets/stylesheets/application.0284f74d.css">
      
        <link rel="stylesheet" href="../assets/stylesheets/application-palette.01803549.css">
      
      
        
        
        <meta name="theme-color" content="#00bcd4">
      
    
    
      <script src="../assets/javascripts/modernizr.74668098.js"></script>
    
    
      
        <link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
        <style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
      
    
    <link rel="stylesheet" href="../assets/fonts/material-icons.css">
    
    
    
      
    
    
  </head>
  
    
    
    <body dir="ltr" data-md-color-primary="cyan" data-md-color-accent="cyan">
  
    <svg class="md-svg">
      <defs>
        
        
      </defs>
    </svg>
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
    
      <a href="#wmt" tabindex="1" class="md-skip">
        Skip to content
      </a>
    
    
      <header class="md-header" data-md-component="header">
  <nav class="md-header-nav md-grid">
    <div class="md-flex">
      <div class="md-flex__cell md-flex__cell--shrink">
        <a href=".." title="sotabencheval Docs" class="md-header-nav__button md-logo">
          
            <i class="md-icon"></i>
          
        </a>
      </div>
      <div class="md-flex__cell md-flex__cell--shrink">
        <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
      </div>
      <div class="md-flex__cell md-flex__cell--stretch">
        <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
          
            <span class="md-header-nav__topic">
              sotabencheval Docs
            </span>
            <span class="md-header-nav__topic">
              
                WMT
              
            </span>
          
        </div>
      </div>
      <div class="md-flex__cell md-flex__cell--shrink">
        
          <label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
          
<div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
      <label class="md-icon md-search__icon" for="__search"></label>
      <button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
        &#xE5CD;
      </button>
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" data-md-scrollfix>
        <div class="md-search-result" data-md-component="result">
          <div class="md-search-result__meta">
            Type to start searching
          </div>
          <ol class="md-search-result__list"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
        
      </div>
      
    </div>
  </nav>
</header>
    
    <div class="md-container">
      
        
      
      
      <main class="md-main">
        <div class="md-main__inner md-grid" data-md-component="container">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    <nav class="md-nav md-nav--primary" data-md-level="0">
  <label class="md-nav__title md-nav__title--site" for="__drawer">
    <a href=".." title="sotabencheval Docs" class="md-nav__button md-logo">
      
        <i class="md-icon"></i>
      
    </a>
    sotabencheval Docs
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
      


  <li class="md-nav__item">
    <a href=".." title="Welcome to sotabencheval!" class="md-nav__link">
      Welcome to sotabencheval!
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../ade20k/" title="ADE20K" class="md-nav__link">
      ADE20K
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../coco/" title="COCO" class="md-nav__link">
      COCO
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../imagenet/" title="ImageNet" class="md-nav__link">
      ImageNet
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../pascalvoc/" title="PASCAL VOC 2012" class="md-nav__link">
      PASCAL VOC 2012
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../squad/" title="SQuAD" class="md-nav__link">
      SQuAD
    </a>
  </li>

    
      
      
      

  


  <li class="md-nav__item md-nav__item--active">
    
    <input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
    
      
    
    
      <label class="md-nav__link md-nav__link--active" for="__toc">
        WMT
      </label>
    
    <a href="./" title="WMT" class="md-nav__link md-nav__link--active">
      WMT
    </a>
    
      
<nav class="md-nav md-nav--secondary">
  
  
    
  
  
    <label class="md-nav__title" for="__toc">Table of contents</label>
    <ul class="md-nav__list" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#getting-started" title="Getting Started" class="md-nav__link">
    Getting Started
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-initialize-an-evaluator" title="How Do I Initialize an Evaluator?" class="md-nav__link">
    How Do I Initialize an Evaluator?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-evaluate-predictions" title="How Do I Evaluate Predictions?" class="md-nav__link">
    How Do I Evaluate Predictions?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-cache-evaluation" title="How Do I Cache Evaluation?" class="md-nav__link">
    How Do I Cache Evaluation?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#a-full-sotabenchpy-example" title="A Full sotabench.py Example" class="md-nav__link">
    A Full sotabench.py Example
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#need-more-help" title="Need More Help?" class="md-nav__link">
    Need More Help?
  </a>
  
</li>
      
      
      
      
      
    </ul>
  
</nav>
    
  </li>

    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    
<nav class="md-nav md-nav--secondary">
  
  
    
  
  
    <label class="md-nav__title" for="__toc">Table of contents</label>
    <ul class="md-nav__list" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#getting-started" title="Getting Started" class="md-nav__link">
    Getting Started
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-initialize-an-evaluator" title="How Do I Initialize an Evaluator?" class="md-nav__link">
    How Do I Initialize an Evaluator?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-evaluate-predictions" title="How Do I Evaluate Predictions?" class="md-nav__link">
    How Do I Evaluate Predictions?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-cache-evaluation" title="How Do I Cache Evaluation?" class="md-nav__link">
    How Do I Cache Evaluation?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#a-full-sotabenchpy-example" title="A Full sotabench.py Example" class="md-nav__link">
    A Full sotabench.py Example
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#need-more-help" title="Need More Help?" class="md-nav__link">
    Need More Help?
  </a>
  
</li>
      
      
      
      
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
          <div class="md-content">
            <article class="md-content__inner md-typeset">
              
                
                
                <h1 id="wmt">WMT</h1>
<p>You can view the WMT Machine Translation leaderboards:</p>
<ul>
<li><a href="https://sotabench.com/benchmarks/machine-translation-on-wmt2014-english-german">WMT2014 English-German</a></li>
<li><a href="https://sotabench.com/benchmarks/machine-translation-on-wmt2014-english-french">WMT2014 English-French</a></li>
<li><a href="https://sotabench.com/benchmarks/machine-translation-on-wmt2019-english-german">WMT2019 English-German</a></li>
</ul>
<h2 id="getting-started">Getting Started</h2>
<p>You'll need the following in the root of your repository:</p>
<ul>
<li><code>sotabench.py</code> file - contains benchmarking logic; the server will run this on each commit</li>
<li><code>requirements.txt</code> file - Python dependencies to be installed before running <code>sotabench.py</code></li>
<li><code>sotabench_setup.sh</code> <em>(optional)</em> - any advanced dependencies or setup, e.g. compilation</li>
</ul>
<p>You can write whatever you want in your <code>sotabench.py</code> file to get model predictions on the WMT datasets.</p>
<p>But you will need to record your results for the server, and you'll want to avoid doing things like
downloading the dataset on the server. So you should:</p>
<ul>
<li><strong>Include an Evaluation object</strong> in <code>sotabench.py</code> file to record the results.</li>
<li><strong>Point to the server WMT data path</strong> - popular datasets are pre-downloaded on the server.</li>
<li><strong>Use Caching</strong> <em>(optional)</em> - to speed up evaluation by hashing the first batch of predictions.</li>
</ul>
<p>We explain how to do these various steps below.</p>
<h2 id="how-do-i-initialize-an-evaluator">How Do I Initialize an Evaluator?</h2>
<p>Before you start batching over the dataset and making predictions you need
to create an evaluator instance to record results for a given leaderboard.
For example, to evaluate on WMT2014 News English-French test set add this
to your code:</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">sotabencheval.machine_translation</span> <span class="kn">import</span> <span class="n">WMTEvaluator</span><span class="p">,</span> <span class="n">WMTDataset</span><span class="p">,</span> <span class="n">Language</span>

<span class="n">evaluator</span> <span class="o">=</span> <span class="n">WMTEvaluator</span><span class="p">(</span>
    <span class="n">dataset</span><span class="o">=</span><span class="n">WMTDataset</span><span class="o">.</span><span class="n">News2014</span><span class="p">,</span>
    <span class="n">source_lang</span><span class="o">=</span><span class="n">Language</span><span class="o">.</span><span class="n">English</span><span class="p">,</span>
    <span class="n">target_lang</span><span class="o">=</span><span class="n">Language</span><span class="o">.</span><span class="n">French</span><span class="p">,</span>
    <span class="n">local_root</span><span class="o">=</span><span class="s1">&#39;mydatasets&#39;</span><span class="p">,</span>
    <span class="n">model_name</span><span class="o">=</span><span class="s1">&#39;My Super Model&#39;</span>
<span class="p">)</span>
</pre></div>


<p>You can use <code>evaluator.source_dataset_path: Path</code> and <code>evaluator.target_dataset_path: Path</code>
to get paths to the source and target SGML files.
In the example above the first one resolves to <code>.data/nlp/wmt/newstest2014-fren-src.en.sgm</code> on
sotabench server and <code>mydatasets/newstest2014-fren-src.en.sgm</code> when run locally.
If you want to use non-standard file names locally you can override the defaults like this:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span> <span class="o">=</span> <span class="n">WMTEvaluator</span><span class="p">(</span>
    <span class="o">...</span><span class="p">,</span>
    <span class="n">local_root</span><span class="o">=</span><span class="s1">&#39;mydatasets&#39;</span>
    <span class="n">source_dataset_filename</span><span class="o">=</span><span class="s1">&#39;english.sgm&#39;</span><span class="p">,</span>
    <span class="n">target_dataset_filename</span><span class="o">=</span><span class="s1">&#39;french.sgm&#39;</span>
<span class="p">)</span>
</pre></div>


<p>If you are reproducing a model from a paper, then you can enter the arXiv ID. If you
put in the same model name string as on the leaderboard
then you will enable direct comparison with the paper's model. For example:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span> <span class="o">=</span> <span class="n">WMTEvaluator</span><span class="p">(</span>
    <span class="n">dataset</span><span class="o">=</span><span class="n">WMTDataset</span><span class="o">.</span><span class="n">News2019</span><span class="p">,</span>
    <span class="n">source_lang</span><span class="o">=</span><span class="n">Language</span><span class="o">.</span><span class="n">English</span><span class="p">,</span>
    <span class="n">target_lang</span><span class="o">=</span><span class="n">Language</span><span class="o">.</span><span class="n">German</span><span class="p">,</span>
    <span class="n">local_root</span><span class="o">=</span><span class="s2">&quot;mydatasets&quot;</span><span class="p">,</span>
    <span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;Facebook-FAIR (single)&quot;</span><span class="p">,</span>
    <span class="n">paper_arxiv_id</span><span class="o">=</span><span class="s2">&quot;1907.06616&quot;</span>
<span class="p">)</span>
</pre></div>


<p>The above will directly compare with the result of the paper when run on the server.</p>
<p>Instead of parsing the dataset files by yourself you can access raw segments as strings:</p>
<div class="codehilite"><pre><span></span>    <span class="k">for</span> <span class="n">segment_id</span><span class="p">,</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">source_segments</span><span class="p">:</span>
        <span class="c1"># translate text</span>

    <span class="c1"># or get segments within document context</span>
    <span class="k">for</span> <span class="n">document</span> <span class="ow">in</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">source_documents</span><span class="p">:</span>
        <span class="n">context</span> <span class="o">=</span> <span class="p">[</span><span class="n">segment</span><span class="o">.</span><span class="n">text</span> <span class="k">for</span> <span class="n">segment</span> <span class="ow">in</span> <span class="n">document</span><span class="o">.</span><span class="n">segments</span><span class="p">]</span>
        <span class="k">for</span> <span class="n">segment</span> <span class="ow">in</span> <span class="n">document</span><span class="o">.</span><span class="n">segments</span><span class="p">:</span>
            <span class="n">segment_id</span><span class="p">,</span> <span class="n">text</span> <span class="o">=</span> <span class="n">segment</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">segment</span><span class="o">.</span><span class="n">text</span>
            <span class="c1"># translate text in context</span>
</pre></div>


<h2 id="how-do-i-evaluate-predictions">How Do I Evaluate Predictions?</h2>
<p>The evaluator object has an <code>.add(answers: Dict[str, str])</code> method to submit predictions by batch or in full.</p>
<p>For WMT the expected input is a dictionary, where keys are source segments
ids and values are translated segments
(segment id is created by concatenating document id and the original segment id,
separted by <code>#</code>.) For example:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">add</span><span class="p">({</span>
    <span class="s1">&#39;bbc.381790#1&#39;</span><span class="p">:</span> <span class="s1">&#39;Waliser AMs sorgen sich um &quot;Aussehen wie Muppets&quot;&#39;</span><span class="p">,</span>
    <span class="s1">&#39;bbc.381790#2&#39;</span><span class="p">:</span> <span class="s1">&#39;Unter einigen AMs herrscht Bestürzung über einen...&#39;</span><span class="p">,</span>
    <span class="s1">&#39;bbc.381790#3&#39;</span><span class="p">:</span> <span class="s1">&#39;Sie ist aufgrund von Plänen entstanden, den Namen...&#39;</span>
<span class="p">})</span>
</pre></div>


<p>You can do this all at once in a single call to <code>add()</code>, but more naturally, you will
probably loop over the dataset and call the method for the outputs of each batch.
That would look something like this (for a PyTorch example):</p>
<div class="codehilite"><pre><span></span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data_loader</span><span class="p">):</span>
        <span class="o">...</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="c1"># potentially formatting of the output here to be a dict</span>
        <span class="n">evaluator</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
</pre></div>


<p>When you are done, you can get the results locally by running:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">get_results</span><span class="p">()</span>
</pre></div>


<p>But for the server you want to save the results by running:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
</pre></div>


<p>This method serialises the results and model metadata and stores to the server database.</p>
<h2 id="how-do-i-cache-evaluation">How Do I Cache Evaluation?</h2>
<p>Sotabench reruns your script on every commit. This is good because it acts like
continuous integration in checking for bugs and changes, but can be annoying
if the model hasn't changed and evaluation is lengthy.</p>
<p>Fortunately sotabencheval has caching logic that you can use.</p>
<p>The idea is that after the first batch, we hash the model outputs and the
current metrics and this tells us if the model is the same given the dataset.
You can include hashing within an evaluation loop like follows (in the following
example for a PyTorch repository):</p>
<div class="codehilite"><pre><span></span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data_loader</span><span class="p">):</span>
        <span class="o">...</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="c1"># potentially formatting of the output here to be a list of dicts</span>
        <span class="n">evaluator</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">cache_exists</span><span class="p">:</span>
            <span class="k">break</span>

<span class="n">evaluator</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
</pre></div>


<p>If the hash is the same as in the server, we infer that the model hasn't changed, so
we simply return hashed results rather than running the whole evaluation again.</p>
<p>Caching is very useful if you have large models, or a repository that is evaluating
multiple models, as it speeds up evaluation significantly.</p>
<h2 id="a-full-sotabenchpy-example">A Full sotabench.py Example</h2>
<p>Below we show an implementation for a model from the torchhub repository. This
incorporates all the features explained above: (a) using the WMT Evaluator,
(b) accessing segments from evaluator, and (c) the evaluation caching logic.
For clarity we omit batching and simply translate segment by segment.</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">sotabencheval.machine_translation</span> <span class="kn">import</span> <span class="n">WMTEvaluator</span><span class="p">,</span> <span class="n">WMTDataset</span><span class="p">,</span> <span class="n">Language</span>
<span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">tqdm</span>
<span class="kn">import</span> <span class="nn">torch</span>

<span class="n">evaluator</span> <span class="o">=</span> <span class="n">WMTEvaluator</span><span class="p">(</span>
    <span class="n">dataset</span><span class="o">=</span><span class="n">WMTDataset</span><span class="o">.</span><span class="n">News2019</span><span class="p">,</span>
    <span class="n">source_lang</span><span class="o">=</span><span class="n">Language</span><span class="o">.</span><span class="n">English</span><span class="p">,</span>
    <span class="n">target_lang</span><span class="o">=</span><span class="n">Language</span><span class="o">.</span><span class="n">German</span><span class="p">,</span>
    <span class="n">local_root</span><span class="o">=</span><span class="s2">&quot;data/nlp/wmt&quot;</span><span class="p">,</span>
    <span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;Facebook-FAIR (single)&quot;</span><span class="p">,</span>
    <span class="n">paper_arxiv_id</span><span class="o">=</span><span class="s2">&quot;1907.06616&quot;</span>
<span class="p">)</span>

<span class="n">model</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">hub</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">&#39;pytorch/fairseq&#39;</span><span class="p">,</span> <span class="s1">&#39;transformer.wmt19.en-de.single_model&#39;</span><span class="p">,</span>
    <span class="n">force_reload</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">tokenizer</span><span class="o">=</span><span class="s1">&#39;moses&#39;</span><span class="p">,</span> <span class="n">bpe</span><span class="o">=</span><span class="s1">&#39;fastbpe&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>

<span class="k">for</span> <span class="n">sid</span><span class="p">,</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">evaluator</span><span class="o">.</span><span class="n">metrics</span><span class="o">.</span><span class="n">source_segments</span><span class="o">.</span><span class="n">items</span><span class="p">()):</span>
    <span class="n">translated</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">translate</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
    <span class="n">evaluator</span><span class="o">.</span><span class="n">add</span><span class="p">({</span><span class="n">sid</span><span class="p">:</span> <span class="n">translated</span><span class="p">})</span>
    <span class="k">if</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">cache_exists</span><span class="p">:</span>
        <span class="k">break</span>

<span class="n">evaluator</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
<span class="k">print</span><span class="p">(</span><span class="n">evaluator</span><span class="o">.</span><span class="n">results</span><span class="p">)</span>
</pre></div>


<h2 id="need-more-help">Need More Help?</h2>
<p>Head on over to the <a href="https://forum.sotabench.com/c/nlp">Natural Language Processing</a> section of the sotabench
forums if you have any questions or difficulties.</p>
                
                  
                
              
              
                


              
            </article>
          </div>
        </div>
      </main>
      
        
<footer class="md-footer">
  
    <div class="md-footer-nav">
      <nav class="md-footer-nav__inner md-grid">
        
          <a href="../squad/" title="SQuAD" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
            <div class="md-flex__cell md-flex__cell--shrink">
              <i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
            </div>
            <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
              <span class="md-flex__ellipsis">
                <span class="md-footer-nav__direction">
                  Previous
                </span>
                SQuAD
              </span>
            </div>
          </a>
        
        
      </nav>
    </div>
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-footer-copyright">
        
        powered by
        <a href="https://www.mkdocs.org">MkDocs</a>
        and
        <a href="https://squidfunk.github.io/mkdocs-material/">
          Material for MkDocs</a>
      </div>
      
    </div>
  </div>
</footer>
      
    </div>
    
      <script src="../assets/javascripts/application.245445c6.js"></script>
      
      <script>app.initialize({version:"1.0.4",url:{base:".."}})</script>
      
    
  </body>
</html>